#!/usr/bin/python
# encoding:utf8
# cosine 2010-11-24

"""
"""

from kscomm.spider.start import Spider

def func(argv_dict):
	print argv_dict['url'],argv_dict['deep']

if __name__ == '__main__':
	# 测试的目标地址，需要Form认证
	url = 'http://t.sina.com.cn/'
	login_url = 'http://login.sina.com.cn/sso/login.php'
	username = 'eniyyy@gmail.com'
	password = '1234567'
	referer = 'http://t.sina.com.cn/'
	user_agent = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)'

	s = Spider(url,deep=2,thread_num=5,crawl_scope=1,url_scope=1,url_type=1,crawl_tags=[],url_tags=['a'])
	s.set_headers(referer=referer,user_agent=user_agent)
	s.set_form_auth(login_url,{'username':username,'password':password})
	s.set_callback(func,['url','deep'])
	s.start()
	s.get_time_statistic()

